{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacked Bidirectional LSTM in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we *stack* LSTM layers to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import imdb\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n",
    "from keras.layers.wrappers import Bidirectional \n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/stackedLSTM'\n",
    "\n",
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64 \n",
    "n_unique_words = 10000 \n",
    "max_review_length = 200 \n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# LSTM layer architecture:\n",
    "n_lstm_1 = 64 # lower\n",
    "n_lstm_2 = 64 # new!\n",
    "drop_lstm = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n",
    "model.add(SpatialDropout1D(drop_embed))\n",
    "model.add(Bidirectional(LSTM(n_lstm_1, dropout=drop_lstm, return_sequences=True))) # retain temporal dimension\n",
    "model.add(Bidirectional(LSTM(n_lstm_2, dropout=drop_lstm)))\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 200, 64)           640000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 200, 64)           0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200, 128)          66048     \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 128)               98816     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 804,993\n",
      "Trainable params: 804,993\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# LSTM layer parameters double due to both reading directions\n",
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 216s - loss: 0.4400 - acc: 0.7801 - val_loss: 0.3231 - val_acc: 0.8677\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 211s - loss: 0.2476 - acc: 0.9040 - val_loss: 0.2927 - val_acc: 0.8764\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 213s - loss: 0.1861 - acc: 0.9310 - val_loss: 0.3139 - val_acc: 0.8680\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 214s - loss: 0.1465 - acc: 0.9461 - val_loss: 0.3704 - val_acc: 0.8610\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f83bb875710>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 87.6% validation accuracy in epoch 2\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 [==============================] - 154s   \n"
     ]
    }
   ],
   "source": [
    "y_hat = model.predict_proba(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFS5JREFUeJzt3X+QXeV93/H3x8iQ2LEtAYKhklrhsewaZ8Y23QG5nkkd\nyxUCZxB/QCNPExRGrTopza9mWuP2D1qwZ3B/kTKT4KpBjfAklgmNi8amoQo/xm2nYBZDsIEwWgOB\njSjaIJCbUuNAvv3jPsIL2dXelXbvenner5mde873POec52HFfu75cc9NVSFJ6s9blroDkqSlYQBI\nUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOrVimEZJfgX4e0AB3wKuAM4C9gKnAt8E\nfraqvp/kFOBm4G8AzwM/XVVPte18BtgBvAr8YlXdcaz9nn766bV+/fr5j0pabN99fPD6zvctbT+k\nGTzwwAN/WlWr52o3ZwAkWQP8InBOVf2/JLcA24CLgOuram+SLzD4w35je32hqt6TZBvweeCnk5zT\n1vsA8FeAP0jy3qp6dbZ9r1+/nvHx8TkHK43cH3xs8PqJe5ayF9KMkvzxMO2GPQW0AvjRJCuAtwHP\nAh8Hbm3L9wCXtOmtbZ62fFOStPreqnq5qp4EJoDzhty/JGmBzRkAVfUnwL8Bnmbwh/8I8ADwYlW9\n0ppNAmva9BrgmbbuK639adPrM6wjSRqxOQMgySoG797PZnDq5u3AhTM0PfpY0cyybLb6G/e3M8l4\nkvGpqam5uidJOk7DnAL6BPBkVU1V1Z8Dvwf8TWBlOyUEsBY42KYngXUAbfm7gMPT6zOs85qq2lVV\nY1U1tnr1nNcwJEnHaZgAeBrYmORt7Vz+JuBR4G7g0tZmO3Bbm97X5mnL76rBlw7sA7YlOSXJ2cAG\n4BsLMwxJ0nzNeRdQVd2X5FYGt3q+AjwI7AK+BuxN8tlWu6mtchPwxSQTDN75b2vbeaTdQfRo286V\nx7oDSJK0uPLD/I1gY2Nj5W2g+qHkbaD6IZbkgaoam6udnwSWpE4ZAJLUqaEeBbFcrb/qa0uy36eu\n++SS7FeS5sMjAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAk\nqVMGgCR1ygCQpE4ZAJLUKQNAkjo1ZwAkeV+Sh6b9fDfJLyc5Ncn+JAfa66rWPkluSDKR5OEk507b\n1vbW/kCS7bPvVZK02Ib5UvjHgQ8BJDkJ+BPgK8BVwJ1VdV2Sq9r8p4ELgQ3t53zgRuD8JKcCVwNj\nQAEPJNlXVS8s+KgkaQEs1ZdKwWi+WGq+p4A2Ad+pqj8GtgJ7Wn0PcEmb3grcXAP3AiuTnAVcAOyv\nqsPtj/5+YMsJj0CSdFzmGwDbgC+16TOr6lmA9npGq68Bnpm2zmSrzVaXJC2BoQMgycnAxcDvztV0\nhlodo/7G/exMMp5kfGpqatjuSZLmaT5HABcC36yq59r8c+3UDu31UKtPAuumrbcWOHiM+utU1a6q\nGquqsdWrV8+je5Kk+ZhPAHyKH5z+AdgHHL2TZztw27T65e1uoI3AkXaK6A5gc5JV7Y6hza0mSVoC\nc94FBJDkbcDfBv7BtPJ1wC1JdgBPA5e1+u3ARcAE8BJwBUBVHU5yLXB/a3dNVR0+4RFIko7LUAFQ\nVS8Bp72h9jyDu4Le2LaAK2fZzm5g9/y7KUlaaH4SWJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXK\nAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSp4YK\ngCQrk9ya5I+SPJbkI0lOTbI/yYH2uqq1TZIbkkwkeTjJudO2s721P5Bk++x7lCQttmGPAP498PtV\n9deBDwKPAVcBd1bVBuDONg9wIbCh/ewEbgRIcipwNXA+cB5w9dHQkCSN3pwBkOSdwE8ANwFU1fer\n6kVgK7CnNdsDXNKmtwI318C9wMokZwEXAPur6nBVvQDsB7Ys6GgkSUMb5gjg3cAU8J+SPJjkN5O8\nHTizqp4FaK9ntPZrgGemrT/ZarPVJUlLYJgAWAGcC9xYVR8G/i8/ON0zk8xQq2PUX79ysjPJeJLx\nqampIbonSToewwTAJDBZVfe1+VsZBMJz7dQO7fXQtPbrpq2/Fjh4jPrrVNWuqhqrqrHVq1fPZyyS\npHmYMwCq6n8DzyR5XyttAh4F9gFH7+TZDtzWpvcBl7e7gTYCR9opojuAzUlWtYu/m1tNkrQEVgzZ\n7heA305yMvAEcAWD8LglyQ7gaeCy1vZ24CJgAniptaWqDie5Fri/tbumqg4vyCgkSfM2VABU1UPA\n2AyLNs3QtoArZ9nObmD3fDooSVocfhJYkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS\n1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnhgqAJE8l+VaS\nh5KMt9qpSfYnOdBeV7V6ktyQZCLJw0nOnbad7a39gSTbZ9ufJGnxzecI4Cer6kNVdfS7ga8C7qyq\nDcCdbR7gQmBD+9kJ3AiDwACuBs4HzgOuPhoakqTRO5FTQFuBPW16D3DJtPrNNXAvsDLJWcAFwP6q\nOlxVLwD7gS0nsH9J0gkYNgAK+G9JHkiys9XOrKpnAdrrGa2+Bnhm2rqTrTZb/XWS7EwynmR8ampq\n+JFIkuZlxZDtPlpVB5OcAexP8kfHaJsZanWM+usLVbuAXQBjY2N/abkkaWEMdQRQVQfb6yHgKwzO\n4T/XTu3QXg+15pPAummrrwUOHqMuSVoCcwZAkrcnecfRaWAz8G1gH3D0Tp7twG1teh9websbaCNw\npJ0iugPYnGRVu/i7udUkSUtgmFNAZwJfSXK0/e9U1e8nuR+4JckO4Gngstb+duAiYAJ4CbgCoKoO\nJ7kWuL+1u6aqDi/YSCRJ8zJnAFTVE8AHZ6g/D2yaoV7AlbNsazewe/7dlCQtND8JLEmdMgAkqVMG\ngCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBI\nUqcMAEnqlAEgSZ0aOgCSnJTkwSRfbfNnJ7kvyYEkX05ycquf0uYn2vL107bxmVZ/PMkFCz0YSdLw\n5nME8EvAY9PmPw9cX1UbgBeAHa2+A3ihqt4DXN/akeQcYBvwAWAL8BtJTjqx7kuSjtdQAZBkLfBJ\n4DfbfICPA7e2JnuAS9r01jZPW76ptd8K7K2ql6vqSQZfGn/eQgxCkjR/wx4B/BrwT4G/aPOnAS9W\n1SttfhJY06bXAM8AtOVHWvvX6jOsI0kasTkDIMlPAYeq6oHp5Rma1hzLjrXO9P3tTDKeZHxqamqu\n7kmSjtMwRwAfBS5O8hSwl8Gpn18DViZZ0dqsBQ626UlgHUBb/i7g8PT6DOu8pqp2VdVYVY2tXr16\n3gOSJA1nzgCoqs9U1dqqWs/gIu5dVfV3gbuBS1uz7cBtbXpfm6ctv6uqqtW3tbuEzgY2AN9YsJFI\nkuZlxdxNZvVpYG+SzwIPAje1+k3AF5NMMHjnvw2gqh5JcgvwKPAKcGVVvXoC+5cknYB5BUBV3QPc\n06afYIa7eKrqe8Bls6z/OeBz8+2kJGnh+UlgSeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6\nZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdmjMAkvxI\nkm8k+cMkjyT5l61+dpL7khxI8uUkJ7f6KW1+oi1fP21bn2n1x5NcsFiDkiTNbZgjgJeBj1fVB4EP\nAVuSbAQ+D1xfVRuAF4Adrf0O4IWqeg9wfWtHknMYfEH8B4AtwG8kOWkhByNJGt6cAVADf9Zm39p+\nCvg4cGur7wEuadNb2zxt+aYkafW9VfVyVT0JTDDDl8pLkkZjqGsASU5K8hBwCNgPfAd4sapeaU0m\ngTVteg3wDEBbfgQ4bXp9hnUkSSM2VABU1atV9SFgLYN37e+fqVl7zSzLZqu/TpKdScaTjE9NTQ3T\nPUnScZjXXUBV9SJwD7ARWJlkRVu0FjjYpieBdQBt+buAw9PrM6wzfR+7qmqsqsZWr149n+5JkuZh\nmLuAVidZ2aZ/FPgE8BhwN3Bpa7YduK1N72vztOV3VVW1+rZ2l9DZwAbgGws1EEnS/KyYuwlnAXva\nHTtvAW6pqq8meRTYm+SzwIPATa39TcAXk0wweOe/DaCqHklyC/Ao8ApwZVW9urDDkSQNa84AqKqH\ngQ/PUH+CGe7iqarvAZfNsq3PAZ+bfzclSQvNTwJLUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTg3z\nOQBJWlLrr/raUnfhTckjAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQB\nIEmdMgAkqVPDfCn8uiR3J3ksySNJfqnVT02yP8mB9rqq1ZPkhiQTSR5Ocu60bW1v7Q8k2T7bPiVJ\ni2+YI4BXgF+tqvcDG4Erk5wDXAXcWVUbgDvbPMCFwIb2sxO4EQaBAVwNnM/gu4SvPhoakqTRmzMA\nqurZqvpmm/4/wGPAGmArsKc12wNc0qa3AjfXwL3AyiRnARcA+6vqcFW9AOwHtizoaCRJQ5vXNYAk\n64EPA/cBZ1bVszAICeCM1mwN8My01SZbbbb6G/exM8l4kvGpqan5dE+SNA9DB0CSHwP+M/DLVfXd\nYzWdoVbHqL++ULWrqsaqamz16tXDdk+SNE9DBUCStzL44//bVfV7rfxcO7VDez3U6pPAummrrwUO\nHqMuSVoCw9wFFOAm4LGq+nfTFu0Djt7Jsx24bVr98nY30EbgSDtFdAewOcmqdvF3c6tJkpbAMF8J\n+VHgZ4FvJXmo1f4ZcB1wS5IdwNPAZW3Z7cBFwATwEnAFQFUdTnItcH9rd01VHV6QUUiS5m3OAKiq\n/8HM5+8BNs3QvoArZ9nWbmD3fDooSVocfhJYkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdWqYzwFo\nntZf9bUl2e9T131ySfYraXnyCECSOmUASFKnPAUkaWhLdXpTi8MjAEnqlAEgSZ0yACSpUwaAJHXK\nAJCkThkAktQpA0CSOjXMdwLvTnIoyben1U5Nsj/Jgfa6qtWT5IYkE0keTnLutHW2t/YHkmyfaV+S\npNEZ5gjgt4Atb6hdBdxZVRuAO9s8wIXAhvazE7gRBoEBXA2cD5wHXH00NCRJS2OY7wT+epL1byhv\nBT7WpvcA9wCfbvWb2/cC35tkZZKzWtv9R78EPsl+BqHypRMegV6zlJ/S9EF00vJzvNcAzqyqZwHa\n6xmtvgZ4Zlq7yVabrS5JWiILfRE4M9TqGPW/vIFkZ5LxJONTU1ML2jlJ0g8c78PgnktyVlU9207x\nHGr1SWDdtHZrgYOt/rE31O+ZacNVtQvYBTA2NjZjSEg984FsWijHewSwDzh6J8924LZp9cvb3UAb\ngSPtFNEdwOYkq9rF382tJklaInMeAST5EoN376cnmWRwN891wC1JdgBPA5e15rcDFwETwEvAFQBV\ndTjJtcD9rd01Ry8IS5KWxjB3AX1qlkWbZmhbwJWzbGc3sHtevdOy0ePXYN77xPNs83SMljG/EEbL\n2lIFz953P78k+5UWko+CkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkA\nktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE6NPACSbEnyeJKJJFeNev+SpIGRBkCS\nk4BfBy4EzgE+leScUfZBkjQw6iOA84CJqnqiqr4P7AW2jrgPkiRGHwBrgGemzU+2miRpxFaMeH+Z\noVava5DsBHa22T9L8vhx7ut04E+Pc93lqscxwxKM+yOvTf3UKHc7XY+/667GnM8Dxz/mvzZMo1EH\nwCSwbtr8WuDg9AZVtQvYdaI7SjJeVWMnup3lpMcxQ5/jdsx9WOwxj/oU0P3AhiRnJzkZ2AbsG3Ef\nJEmM+Aigql5J8o+AO4CTgN1V9cgo+yBJGhj1KSCq6nbg9hHs6oRPIy1DPY4Z+hy3Y+7Doo45VTV3\nK0nSm46PgpCkTi37AJjr0RJJTkny5bb8viTrR9/LhTXEmP9xkkeTPJzkziRD3RL2w2zYR4gkuTRJ\nJXlT3C0yzLiT/J32+34kye+Muo8LbYh/3381yd1JHmz/xi9ain4upCS7kxxK8u1ZlifJDe2/ycNJ\nzl2QHVfVsv1hcCH5O8C7gZOBPwTOeUObfwh8oU1vA7681P0ewZh/Enhbm/75Hsbc2r0D+DpwLzC2\n1P0e0e96A/AgsKrNn7HU/R7BmHcBP9+mzwGeWup+L8C4fwI4F/j2LMsvAv4rg89SbQTuW4j9Lvcj\ngGEeLbEV2NOmbwU2JZnpA2nLxZxjrqq7q+qlNnsvg89bLGfDPkLkWuBfAd8bZecW0TDj/vvAr1fV\nCwBVdWjEfVxow4y5gHe26Xfxhs8SLUdV9XXg8DGabAVuroF7gZVJzjrR/S73ABjm0RKvtamqV4Aj\nwGkj6d3imO/jNHYweOewnM055iQfBtZV1VdH2bFFNszv+r3Ae5P8zyT3Jtkyst4tjmHG/C+An0ky\nyeCOwl8YTdeW1KI8Rmfkt4EusDkfLTFkm+Vk6PEk+RlgDPhbi9qjxXfMMSd5C3A98HOj6tCIDPO7\nXsHgNNDHGBzp/fckP15VLy5y3xbLMGP+FPBbVfVvk3wE+GIb818sfveWzKL8HVvuRwBzPlpiepsk\nKxgcMh7rUOuH3TBjJskngH8OXFxVL4+ob4tlrjG/A/hx4J4kTzE4R7rvTXAheNh/37dV1Z9X1ZPA\n4wwCYbkaZsw7gFsAqup/AT/C4Jk5b2ZD/X8/X8s9AIZ5tMQ+YHubvhS4q9pVlWVqzjG30yH/gcEf\n/+V+ThjmGHNVHamq06tqfVWtZ3Dd4+KqGl+a7i6YYf59/xcGF/1JcjqDU0JPjLSXC2uYMT8NbAJI\n8n4GATA10l6O3j7g8nY30EbgSFU9e6IbXdangGqWR0skuQYYr6p9wE0MDhEnGLzz37Z0PT5xQ475\nXwM/Bvxuu979dFVdvGSdPkFDjvlNZ8hx3wFsTvIo8CrwT6rq+aXr9YkZcsy/CvzHJL/C4DTIzy3z\nN3Uk+RKD03int2sbVwNvBaiqLzC41nERMAG8BFyxIPtd5v/dJEnHabmfApIkHScDQJI6ZQBIUqcM\nAEnqlAEgSZ0yACSpUwaAJHXKAJCkTv1/xIVdLDz8sI4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f83bdedaac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'94.78'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
